{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-11T08:43:28.604710Z",
     "start_time": "2018-03-11T08:43:27.843460Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"Contains a variant of the densenet model definition.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "\n",
    "\n",
    "def trunc_normal(stddev): return tf.truncated_normal_initializer(stddev=stddev)\n",
    "\n",
    "\n",
    "def bn_act_conv_drp(current, num_outputs, kernel_size, scope='block'):\n",
    "    current = slim.batch_norm(current, scope=scope + '_bn')\n",
    "    current = tf.nn.relu(current)\n",
    "    current = slim.conv2d(current, num_outputs, kernel_size, scope=scope + '_conv')\n",
    "    current = slim.dropout(current, scope=scope + '_dropout')\n",
    "    return current\n",
    "\n",
    "\n",
    "def block(net, layers, growth, scope='block'):\n",
    "    for idx in range(layers):\n",
    "        bottleneck = bn_act_conv_drp(net, 4 * growth, [1, 1],\n",
    "                                     scope=scope + '_conv1x1' + str(idx))\n",
    "        tmp = bn_act_conv_drp(bottleneck, growth, [3, 3],\n",
    "                              scope=scope + '_conv3x3' + str(idx))\n",
    "        net = tf.concat(axis=3, values=[net, tmp])\n",
    "    return net\n",
    "\n",
    "\n",
    "def densenet(images, num_classes=1000, is_training=False,\n",
    "             dropout_keep_prob=0.8,\n",
    "             scope='densenet'):\n",
    "    \"\"\"Creates a variant of the densenet model.\n",
    "\n",
    "      images: A batch of `Tensors` of size [batch_size, height, width, channels].\n",
    "      num_classes: the number of classes in the dataset.\n",
    "      is_training: specifies whether or not we're currently training the model.\n",
    "        This variable will determine the behaviour of the dropout layer.\n",
    "      dropout_keep_prob: the percentage of activation values that are retained.\n",
    "      prediction_fn: a function to get predictions out of logits.\n",
    "      scope: Optional variable_scope.\n",
    "\n",
    "    Returns:\n",
    "      logits: the pre-softmax activations, a tensor of size\n",
    "        [batch_size, `num_classes`]\n",
    "      end_points: a dictionary from components of the network to the corresponding\n",
    "        activation.\n",
    "    \"\"\"\n",
    "    growth = 24\n",
    "    compression_rate = 0.5\n",
    "\n",
    "    def reduce_dim(input_feature):\n",
    "        return int(int(input_feature.shape[-1]) * compression_rate)\n",
    "\n",
    "    end_points = {}\n",
    "\n",
    "    with tf.variable_scope(scope, 'DenseNet', [images, num_classes]):\n",
    "        with slim.arg_scope(bn_drp_scope(is_training=is_training,\n",
    "                                         keep_prob=dropout_keep_prob)) as ssc:\n",
    "\n",
    "            with tf.variable_scope(\"first_conv\"):\n",
    "                end_point = \"first_conv\"\n",
    "                net = slim.conv2d(images, growth * 2, [7,7],scope=end_point + \"_conv7x7\")\n",
    "                net = slim.avg_pool2d(net, [3,3],scope=end_point + \"_pool3x3\")\n",
    "                end_points[end_point] = net\n",
    "             \n",
    "            with tf.variable_scope(\"dense_block_1\"):\n",
    "                end_point = \"dense_block_1\"\n",
    "                net = block(net,6,growth,scope=end_point)\n",
    "                end_points[end_point] = net\n",
    "                with tf.variable_scope(\"transition_layer_1\"):\n",
    "                    end_point = \"transition_layer_1\"\n",
    "                    num_outputs = reduce_dim(net)\n",
    "                    net = transition_layer(net,num_outputs,end_point)\n",
    "                    end_points[end_point] = net\n",
    "                    \n",
    "            with tf.variable_scope(\"dense_block_2\"):\n",
    "                end_point = \"dense_block_2\"\n",
    "                net = block(net,12,growth,scope=end_point)\n",
    "                end_points[end_point] = net\n",
    "                with tf.variable_scope(\"transition_layer_2\"):\n",
    "                    end_point = \"transition_layer_2\"\n",
    "                    num_outputs = reduce_dim(net)\n",
    "                    net = transition_layer(net,num_outputs,end_point)\n",
    "                    end_points[end_point] = net\n",
    "                    \n",
    "            with tf.variable_scope(\"dense_block_3\"):\n",
    "                end_point = \"dense_block_3\"\n",
    "                net = block(net,24,growth,scope=end_point)\n",
    "                end_points[end_point] = net\n",
    "                with tf.variable_scope(\"transition_layer_3\"):\n",
    "                    end_point = \"transition_layer_3\"\n",
    "                    num_outputs = reduce_dim(net)\n",
    "                    net = transition_layer(net,num_outputs,end_point)\n",
    "                    end_points[end_point] = net\n",
    "                    \n",
    "            with tf.variable_scope(\"dense_block_4\"):\n",
    "                end_point = \"dense_block_4\"\n",
    "                net = block(net,16,growth,scope=end_point)\n",
    "                end_points[end_point] = net\n",
    "                        \n",
    "            with slim.arg_scope(densenet_arg_scope()) as ssc:\n",
    "                with tf.variable_scope(\"logits\"):\n",
    "                    end_point = \"logits\"\n",
    "                    logits = classification_layer(net,num_classes,end_point)\n",
    "                    end_points[end_point] = logits\n",
    "    return logits, end_points\n",
    "\n",
    "\n",
    "def bn_drp_scope(is_training=True, keep_prob=0.8):\n",
    "    keep_prob = keep_prob if is_training else 1\n",
    "    with slim.arg_scope(\n",
    "        [slim.batch_norm],\n",
    "            scale=True, is_training=is_training, updates_collections=None):\n",
    "        with slim.arg_scope(\n",
    "            [slim.dropout],\n",
    "                is_training=is_training, keep_prob=keep_prob) as bsc:\n",
    "            return bsc\n",
    "\n",
    "\n",
    "def densenet_arg_scope(weight_decay=0.004):\n",
    "    \"\"\"Defines the default densenet argument scope.\n",
    "\n",
    "    Args:\n",
    "      weight_decay: The weight decay to use for regularizing the model.\n",
    "\n",
    "    Returns:\n",
    "      An `arg_scope` to use for the inception v3 model.\n",
    "    \"\"\"\n",
    "    with slim.arg_scope(\n",
    "        [slim.conv2d],\n",
    "        weights_initializer=tf.contrib.layers.variance_scaling_initializer(\n",
    "            factor=2.0, mode='FAN_IN', uniform=False),\n",
    "        activation_fn=None, biases_initializer=None, padding='same',\n",
    "            stride=1) as sc:\n",
    "        return sc\n",
    "\n",
    "def transition_layer(net,num_outputs,scope=\"transition\"):\n",
    "    net = bn_act_conv_drp(net,num_outputs,[1,1],scope=scope)\n",
    "    net = slim.avg_pool2d(net,[2,2],stride=2,scope=scope + \"_pool2x2\")\n",
    "   \n",
    "    return net\n",
    "\n",
    "def classification_layer(net,num_classes,scope=\"logits\"):\n",
    "    \n",
    "    net = slim.batch_norm(net,scope=scope + \"_bn\")\n",
    "    net = tf.nn.relu(net)\n",
    "    kernel_size = _reduced_kernel_size_for_small_input(net,[7,7])\n",
    "    net = slim.avg_pool2d(net,kernel_size,stride=2,scope=scope + '_pool{}x{}'.format(*kernel_size))         \n",
    "#     net = slim.flatten(net)\n",
    "#     logits = slim.fully_connected(net, num_classes, scope=scope + \"_fn\")\n",
    "    net = slim.conv2d(net,num_classes,[1,1])\n",
    "    net = slim.dropout(net, scope=scope + '_dropout')\n",
    "    logits = tf.squeeze(net, [1, 2], name='squeeze')\n",
    "   \n",
    "    return logits\n",
    "\n",
    "densenet.default_image_size = 32\n",
    "\n",
    "def _reduced_kernel_size_for_small_input(input_tensor, kernel_size):\n",
    "    shape = input_tensor.get_shape().as_list()\n",
    "    if shape[1] is None or shape[2] is None:\n",
    "        kernel_size_out = kernel_size\n",
    "    else:\n",
    "        kernel_size_out = [min(shape[1], kernel_size[0]),\n",
    "                           min(shape[2], kernel_size[1])]\n",
    "    return kernel_size_out\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
